TODO: Revise Jain’s principles and common mistakes.
We used a dataset with 2^24 packets in the detection phase.
log2n = as.integer(24) # n is passed to trafg.
n = as.integer(2^log2n)
pcap_dir = "~/p4sec/ddosm-p4/pcaps"
pcap_csv_m14 = str_c(pcap_dir, "/ddos20m14b/if3_attack_out.csv")
pcap_csv_m16 = str_c(pcap_dir, "/ddos20m16b/if3_attack_out.csv")
pcap_csv_m18 = str_c(pcap_dir, "/ddos20m18b/if3_attack_out.csv")
What are our chosen k coefficients?
Our experiments with tcad_m_levels.py show us candidate values for k, as follows:
| Log2(m) | k | FPR |
|---|---|---|
| 14 | 4.125 | 1.7% |
| 16 | 4.500 | 1.6% |
| 18 | 3.625 | 0.0% |
tcad_m14_k = 4.125
tcad_m16_k = 4.5
tcad_m18_k = 3.625
TCAD measurements can be found in the following files:
tcad_m_2_14_k_4.125.log tcad_m_2_16_k_4.500.log tcad_m_2_18_k_3.625.log
[Source: DDoS Mitigation.ipynb, section Finding TCAD Values]
trace_dir = "~/p4sec/ddosm-p4/lab/ddos20/tcad_logs"
tcad_m14_trace = str_c(trace_dir, "/tcad_m_2_14_k_4.125.log")
tcad_m16_trace = str_c(trace_dir, "/tcad_m_2_16_k_4.500.log")
tcad_m18_trace = str_c(trace_dir, "/tcad_m_2_18_k_3.625.log")
Note: When we preinitialize training coefficients, the length of the workload is equal to the length of the detection phase.
For a detection phase of 2^24 packets we have:
2^(24-log2m-2) OWs before the attack, 2^(24-log2m-1) OWs under attack, 2^(24-log2m-2) OWs after the attack.
For m=2^14, the detection phase has 2(24-14)=210 windows:
2^8 OWs pre-attack and post attack,
2^9 OWs under attack,
For m=2^16, 2^8 windows:
2^6 OWs pre-attack and post-attack, 2^7 OWs under attack.
For m=2^18, 2^6 windows:
2^4 OWs pre-attack and post-attack, 2^5 OWs under attack.
Generally:
# Function inputs are expressed in numbers of packets.
# Log2n: Length of the detection phase, passed to trafg (as
# '-n 1048576', for instance). Log2m: Length of the
# observation window, passed to tcad JSON file (as
# ''window_size': 262144', for instance) Function outputs are
# expressed in numbers of observation windows.
detection = function(log2n, log2m) as.integer(2^(log2n - log2m))
training = function(log2n, log2m) as.integer(2^(log2n - log2m -
1)) # Training length = detection / 2
attack = function(log2n, log2m) as.integer(2^(log2n - log2m -
1)) # Attack length = detection / 2
safety = function(log2n, log2m) as.integer(2^(log2n - log2m -
2)) # Pre-attack and post-attack = attack / 4 (each)
attack_first = function(log2n, log2m) as.integer(training(log2n,
log2m) + safety(log2n, log2m) + 1)
attack_last = function(log2n, log2m) as.integer(training(log2n,
log2m) + safety(log2n, log2m) + attack(log2n, log2m))
# We also define a helper function to get the OW number from
# a packet index.
get_ow = function(index, m) as.integer((index - 1)%/%m + 1)
read_tcad_trace = function(trace_file) {
col_names = c("ts", "src_ent", "src_ewma", "src_ewmmd", "dst_ent",
"dst_ewma", "dst_ewmmd", "alarm")
col_types = "ciddiddl"
tcad_trace = readr::read_table2(trace_file, col_names = col_names,
col_types = col_types)
tcad_trace = tcad_trace %>% tibble::rowid_to_column("ow")
# Entropy values: 4 fractional bits. EWMA/EWMMD: 18
# fractional bits.
tcad_trace = tcad_trace %>% dplyr::mutate(src_ent = src_ent/16,
dst_ent = dst_ent/16, src_ewma = src_ewma/262144, dst_ewma = dst_ewma/262144,
src_ewmmd = src_ewmmd/262144, dst_ewmmd = dst_ewmmd/262144)
return(tcad_trace)
}
tcad_m14 = read_tcad_trace(tcad_m14_trace)
tcad_m16 = read_tcad_trace(tcad_m16_trace)
tcad_m18 = read_tcad_trace(tcad_m18_trace)
get_plot_tcad = function(tcad, k) {
plot_options = list(labs(x = "OW number", y = "Entropy"),
geom_point(mapping = aes(y = src_ent), size = 0.25, color = "seagreen4"),
geom_point(mapping = aes(y = dst_ent), size = 0.25, color = "steelblue4"),
geom_line(mapping = aes(y = src_ewma + k * src_ewmmd),
color = "seagreen4"), geom_line(mapping = aes(y = dst_ewma -
k * dst_ewmmd), color = "steelblue4"), theme_classic())
plot = tcad %>% ggplot(mapping = aes(x = ow)) + plot_options
return(plot)
}
title = "Entropy for each Observation Window "
get_plot_tcad(tcad_m14, tcad_m14_k) + labs(title = str_c(title,
"(m = 2^14)"))
get_plot_tcad(tcad_m16, tcad_m16_k) + labs(title = str_c(title,
"(m = 2^16)"))
get_plot_tcad(tcad_m18, tcad_m18_k) + labs(title = str_c(title,
"(m = 2^18)"))
Alright, we have the graph for the entire experiments. Now we need to focus in the attack.
get_plot_tcad_attack = function(tcad, tcad_k, log2n, log2m) {
first = attack_first(log2n, log2m) + 1
last = attack_last(log2n, log2m)
plot = get_plot_tcad(tcad %>% filter(ow >= first, ow <= last),
tcad_k)
return(plot)
}
title = "Entropy for each Observation Window - Attack Phase "
get_plot_tcad_attack(tcad_m14, tcad_m14_k, log2n, 14) + labs(title = str_c(title,
"(m = 2^14)"))
get_plot_tcad_attack(tcad_m16, tcad_m16_k, log2n, 16) + labs(title = str_c(title,
"(m = 2^16)"))
get_plot_tcad_attack(tcad_m18, tcad_m18_k, log2n, 18) + labs(title = str_c(title,
"(m = 2^18)"))
We begin with log2m = 14.
read_pcap_csv = function(log2n, log2m, pcap_csv) {
m = as.integer(2^log2m)
# This is the format of the CSV files we import.
col_types = cols(src = col_character(), dst = col_character(),
src_delta = col_character(), dst_delta = col_character(),
attack = col_logical())
packets = read_csv(pcap_csv, col_types = col_types)
# Add index column.
packets = packets %>% tibble::rowid_to_column("index")
# Add an offset to compensate the pre-initializing of
# training coefficients.
offset = training(log2n, log2m) * m
packets = packets %>% mutate(index = index + offset)
# Add OW numbers.
packets = packets %>% mutate(ow = get_ow(index, m))
# Adjust numeric representations of src_delta and dst_delta.
# Convert from hexadecimal to decimal
packets = packets %>% mutate_at(vars(src_delta, dst_delta),
funs(strtoi))
# Convert from 16-bit two's complement representation to
# integer representation.
twos_complement = function(x) as.integer(ifelse(x > 32767,
x - 65536, x))
packets = packets %>% mutate_at(vars(src_delta, dst_delta),
funs(twos_complement))
return(packets)
}
packets = read_pcap_csv(log2n = 24, log2m = 14, pcap_csv = pcap_csv_m14)
## Warning: funs() is soft deprecated as of dplyr 0.8.0
## Please use a list of either functions or lambdas:
##
## # Simple named list:
## list(mean = mean, median = median)
##
## # Auto named with `tibble::lst()`:
## tibble::lst(mean, median)
##
## # Using lambdas
## list(~ mean(., trim = .2), ~ median(., na.rm = TRUE))
## This warning is displayed once per session.
At this point we need to check whether we can observe the first attack packet. We query the dataset at a specific points. For log2m=14, the attack begins at the fifth packet of OW 769 and lasts for a total of 512 OWs (i.e., OWs 769-1280).
log2m = as.integer(14)
m = as.integer(2^log2m)
attack_first_ow = attack_first(log2n, log2m)
attack_last_ow = attack_last(log2n, log2m)
attack_first_packet = (attack_first_ow - 1) * m + 1
attack_last_packet = (attack_last_ow) * m
attack_first_ow
## [1] 769
attack_last_ow
## [1] 1280
attack_first_packet
## [1] 12582913
attack_last_packet
## [1] 20971520
packets %>% filter(index >= attack_first_packet)
Question: for each OW, what are the typical frequency deltas for attack packets?
For log2m=14, packet diversion begins at OW 770. DEFCON uses OWs 769 and 768 as a reference.
summarize_deltas = function(log2n, log2m, packets) {
attack_first_ow = attack_first(log2n, log2m) + 1
attack_last_ow = attack_last(log2n, log2m)
result = packets %>% filter(ow >= attack_first_ow, ow <=
attack_last_ow) %>% group_by(ow, attack) %>% summarize(srcq1 = quantile(src_delta,
0.25), srcq2 = median(src_delta), srcq3 = quantile(src_delta,
0.75), srciqr = IQR(src_delta), dstq1 = quantile(dst_delta,
0.25), dstq2 = median(dst_delta), dstq3 = quantile(dst_delta,
0.75), dstiqr = IQR(dst_delta))
return(result)
}
deltas = summarize_deltas(log2n, log2m, packets)
deltas
Can we graph it?
dot_size = 2
delta_plot_options = list(geom_point(mapping = aes(y = srcq1),
color = "blue4", position = "jitter", size = dot_size), geom_point(mapping = aes(y = srcq2),
color = "yellow4", position = "jitter", size = dot_size),
geom_point(mapping = aes(y = srcq3), color = "orangered4",
position = "jitter", size = dot_size))
deltas %>% filter(ow < 800) %>% ggplot(mapping = aes(x = ow,
shape = attack)) + delta_plot_options
deltas %>% filter(ow >= 800) %>% ggplot(mapping = aes(x = ow,
shape = attack)) + delta_plot_options
delta_plot_options = list(geom_point(mapping = aes(y = dstq1),
color = "blue4", position = "jitter", size = dot_size), geom_point(mapping = aes(y = dstq2),
color = "yellow4", position = "jitter", size = dot_size),
geom_point(mapping = aes(y = dstq3), color = "orangered4",
position = "jitter", size = dot_size))
deltas %>% filter(ow < 800) %>% ggplot(mapping = aes(x = ow,
shape = attack)) + delta_plot_options
deltas %>% filter(ow > 800) %>% ggplot(mapping = aes(x = ow,
shape = attack)) + delta_plot_options
Let’s set an arbitrary threshold and a function which indicates whether or not to divert a given packet.
threshold = 16
# divert = function(src_delta, dst_delta) (src_delta >=
# threshold) divert = function(src_delta, dst_delta)
# (dst_delta >= threshold) divert = function(src_delta,
# dst_delta) (src_delta >= threshold && dst_delta >=
# threshold)
divert = function(src_delta, dst_delta) (dst_delta >= threshold)
Get the stats for all but the first two OWs under attack: address counts for each delta value.
src_distinct = packets %>% filter(ow >= 770, ow <= 1280) %>%
group_by(ow, attack, src_delta) %>% summarize(srcs = n_distinct(src))
src_distinct
src_distinct %>% ggplot(mapping = aes(x = src_delta, y = srcs,
color = attack)) + geom_line() + scale_color_manual(values = c("seagreen4",
"orangered1"))
dst_distinct = packets %>% filter(ow >= 770, ow <= 1280) %>%
group_by(ow, attack, dst_delta) %>% summarize(dsts = n_distinct(dst))
dst_distinct
dst_distinct %>% ggplot(mapping = aes(x = dst_delta, y = dsts,
color = attack)) + geom_line() + scale_color_manual(values = c("seagreen4",
"orangered1"))
Base Stats
stats = function(packets, attack_first_ow, attack_last_ow) {
query = packets %>% filter(ow >= attack_first_ow, ow <= attack_last_ow)
true_evil = query %>% filter(attack == TRUE) %>% tally()
true_good = query %>% filter(attack == FALSE) %>% tally()
message("True evil: ", true_evil, " True good: ", true_good,
" Total: ", true_evil + true_good)
class_evil = query %>% filter(divert(src_delta, dst_delta)) %>%
tally()
class_good = query %>% filter(!divert(src_delta, dst_delta)) %>%
tally()
message("Class evil: ", class_evil, " Class good: ", class_good,
" Total: ", class_evil + class_good)
error_evil = query %>% filter(!divert(src_delta, dst_delta),
attack == TRUE) %>% tally()
error_good = query %>% filter(divert(src_delta, dst_delta),
attack == FALSE) %>% tally()
message("FNcount: ", error_evil, " FPcount: ", error_good,
" Total: ", error_evil + error_good)
message("FNR: ", round(error_evil/true_evil, 4), " FPR: ",
round(error_good/true_good, 4))
}
stats(packets, attack_first_ow, attack_last_ow)
## True evil: 1677196 True good: 6711412 Total: 8388608
## Class evil: 2445589 Class good: 5943019 Total: 8388608
## FNcount: 49969 FPcount: 818362 Total: 868331
## FNR: 0.0298 FPR: 0.1219
Confidence Intervals
stats_ci = function(packets, attack_first_ow, attack_last_ow) {
tpr = packets %>% filter(ow >= attack_first_ow, ow <= attack_last_ow,
attack == TRUE, divert(src_delta, dst_delta) == TRUE) %>%
group_by(ow) %>% summarize(n = n()) %>% summarize(mean = mean(n)/(0.2 *
16384), margin = qnorm(0.975) * sd(n)/sqrt(1280 - 770 +
1)/(0.2 * 16384))
fpr = packets %>% filter(ow >= attack_first_ow, ow <= attack_last_ow,
attack == FALSE, divert(src_delta, dst_delta) == TRUE) %>%
group_by(ow) %>% summarize(n = n()) %>% summarize(mean = mean(n)/(0.8 *
16384), margin = qnorm(0.975) * sd(n)/sqrt(1280 - 770 +
1)/(0.8 * 16384))
message(str_c("TPR: ", round(tpr$mean, 6), " ± ", round(tpr$margin,
6)))
message(str_c("FPR: ", round(fpr$mean, 6), " ± ", round(fpr$margin,
6)))
}
stats_ci(packets, attack_first_ow, attack_last_ow)
## TPR: 0.971801 ± 0.002222
## FPR: 0.122184 ± 0.002538
It is interesting to observe what happens over time, OW after OW.
graph_true_good = function() query %>% filter(attack == FALSE) %>%
summarize(n = n()) %>% ggplot(mapping = aes(x = ow, y = n)) +
geom_point() + ggtitle("True Good")
graph_true_evil = function() query %>% filter(attack == TRUE) %>%
summarize(n = n()) %>% ggplot(mapping = aes(x = ow, y = n)) +
geom_point() + ggtitle("True Evil")
graph_class_good = function() query %>% filter(diverted == FALSE) %>%
summarize(n = n()) %>% ggplot(mapping = aes(x = ow, y = n)) +
geom_point() + ggtitle("Forwarded")
graph_class_evil = function() query %>% filter(diverted == TRUE) %>%
summarize(n = n()) %>% ggplot(mapping = aes(x = ow, y = n)) +
geom_point() + ggtitle("Diverted")
graph_false_neg = function() query %>% filter(!divert(src_delta,
dst_delta), attack == TRUE) %>% summarize(n = n()) %>% ggplot(mapping = aes(x = ow,
y = n)) + geom_point() + ggtitle("False Negatives")
graph_false_pos = function() query %>% filter(divert(src_delta,
dst_delta), attack == FALSE) %>% summarize(n = n()) %>% ggplot(mapping = aes(x = ow,
y = n)) + geom_point() + ggtitle("False Positives")
graph_results = function()
query %>% group_by(ow, attack, diverted) %>% summarize(n = n()) %>%
ggplot(mapping = aes(x = ow, y = n, color = attack, shape = diverted)) +
geom_point(position = "jitter", size = 2) + # coord_cartesian(xlim=c(770,810),ylim=c(0,16384)) +
labs(x = "Observation Window", y = "Packet Count", title = "Classification Results") +
scale_x_continuous(expand = expand_scale(add = 0)) + scale_y_continuous(expand = expand_scale(add = 0)) +
scale_color_manual(values = c("seagreen4", "orangered1")) +
theme_classic()
query = packets %>% filter(ow >= attack_first_ow, ow <= attack_last_ow) %>%
mutate(diverted = divert(src_delta, dst_delta)) %>% group_by(ow)
graph_true_good()
graph_true_evil()
graph_class_good()
graph_class_evil()
graph_false_neg()
graph_false_pos()
graph_results()
mitigation_out = packets %>% filter(ow >= attack_first_ow, ow <=
attack_last_ow) %>% mutate(diverted = (src_delta >= threshold)) %>%
group_by(ow, attack, diverted) %>% summarize(n = n())
mitigation_out
If we analyze only the first 30 OWs under attack, with a threshold equal to 16, we divert ~86% of the attack, while keeping ~90% of the good traffic in the original path.
For each OW, how many different attack sources are there?
How do our findings correlate with TCAD measurements?
What else can we do?
rm(query)